In [1]:
import numpy as np
import pickle
import pandas as pd
import glob
import matplotlib.pyplot as plt
from tqdm._tqdm_notebook import tqdm_notebook as tqdm
plt.style.use('seaborn')
import matplotlib.colors as mcolors
cols = mcolors.TABLEAU_COLORS
cols = list(cols.keys())
cols = cols+cols
In [260]:
glob.glob("sims6/*")
Out[260]:
In [ ]:
In [12]:
np.array(a)
np.array(b)
Out[12]:
In [13]:
dat = np.load(FOLDERS[0] + '/SSDF_Simu.npz')
In [14]:
ghg.mean(0)
In [ ]:
incides = [0,1, 3,5,7]
In [2226]:
FOLDERS = sorted(glob.glob('sims6/s*'))
#FOLDERS = [FOLDERS[i] for i in incides]
print(FOLDERS)
def ab(folder):
#dat = np.load(folder + '/SSDF_Simu.npz')
#a = dat['ghg'].mean(0)
ghg =np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/ghg.csv', header=None))
a = np.median(ghg,0)
bhg = np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/bhg.csv', header=None))
b = np.median(bhg,0).reshape(-1,2)[:-1,:] #bhg.mean(0).reshape(-1,2)[:-1,:]
#print(b)
a2 = np.median(ghg,0)#.median(0)
b2 = np.median(bhg,0).reshape(-1,2)[:-1, :]
return a,b, a2, b2
ea, eb, ea2, eb2 = ab(FOLDERS[0])
a = pd.read_csv('sims6/a.csv',index_col=0)
b = pd.read_csv('sims6/b.csv',index_col=0)
delta = [1,-1]
L = a.shape[0]
EP = len(FOLDERS)
ap = np.zeros((EP, L))
bp1 = np.zeros((EP, L-1))
bp2 = np.zeros((EP, L-1))
ap5 = np.zeros((EP, L))
bp15 = np.zeros((EP, L-1))
bp25 = np.zeros((EP, L-1))
for i in tqdm(range(EP)):
try:
ea, eb, ea2, eb2 = ab(FOLDERS[i])
ap[i] = ea
bp1[i] = eb[:,0]
bp2[i] = eb[:,1]
ap5[i] = ea2
bp15[i] = eb2[:,0]
bp25[i] = eb2[:,1]
except:
pass
In [2227]:
#plt.plot(ap)
plt.figure(figsize=(15,10))
plt.subplot(2,2,1)
for i in range(L):
plt.plot(ap[:,i],color=cols[i])
plt.axhline(np.array(a)[i],ls='--', color= cols[i] )
plt.title('A_LOADING', fontsize=15)
#plt.show()
plt.subplot(2,2,3)
#plt.plot(bp1)
for i in range(L-1):
plt.plot(bp1[:,i],color=cols[i])
plt.axhline(np.array(np.array(b)[i,0]),ls='--', color= cols[i])
plt.title('B_LOADING_1', fontsize=15)
#plt.show()
plt.subplot(2,2,4)
#plt.plot(bp2)
for i in range(L-1):
plt.plot(bp2[:,i],color=cols[i])
plt.axhline(np.array(np.array(b)[i,1]),ls='--', color=cols[i])
plt.title('B_LOADING_2', fontsize=15)
#plt.savefig('40sims.png')
plt.show()
In [4]:
FOLDERS = sorted(glob.glob('sims6/archives40/s*'))
#FOLDERS = [FOLDERS[i] for i in incides]
print(FOLDERS)
def ab(folder):
#dat = np.load(folder + '/SSDF_Simu.npz')
#a = dat['ghg'].mean(0)
ghg =np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/ghg.csv', header=None))
a = ghg[-20000:] #np.median(ghg,0)
bhg = np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/bhg.csv', header=None))
b = bhg[-20000:]#np.median(bhg,0).reshape(-1,2)[:-1,:] #bhg.mean(0).reshape(-1,2)[:-1,:]
#print(b)
a2 = np.median(ghg,0)#.median(0)
b2 = np.median(bhg,0).reshape(-1,2)[:-1, :]
return a,b, a2, b2
ea, eb, ea2, eb2 = ab(FOLDERS[0])
a = pd.read_csv('sims6/archives40/a.csv',index_col=0)
b = pd.read_csv('sims6/archives40/b.csv',index_col=0)
delta = [1,-1]
L = a.shape[0]
EP = len(FOLDERS)
ap = np.zeros((EP, 20000, L))
bp1 = np.zeros((EP, 20000, L-1))
bp2 = np.zeros((EP, 20000, L-1))
ap5 = np.zeros((EP, L))
bp15 = np.zeros((EP, L-1))
bp25 = np.zeros((EP, L-1))
for i in tqdm(range(EP)):
try:
ea, eb, ea2, eb2 = ab(FOLDERS[i])
ap[i] = ea
bp1[i] = eb[:,[i*2 for i in range(L-1)]]
bp2[i] = eb[:,[i*2+1 for i in range(L-1)]]
ap5[i] = ea2
bp15[i] = eb2[:,0]
bp25[i] = eb2[:,1]
except:
pass
In [27]:
plt.boxplot( ap.mean(1) )
plt.plot(a,ls='',marker="o",markersize=10, color="red")
plt.show()
In [ ]:
In [29]:
b
Out[29]:
In [40]:
#plt.plot(ap)
plt.figure(figsize=(15,10))
plt.subplot(2,2,1)
plt.boxplot(ap.mean(1))
plt.plot(a,ls='',marker="o",markersize=10, color="red")
#for i in range(L):
# plt.boxplot(ap[:,i],color=cols[i])
# plt.axhline(np.array(a)[i],ls='--', color= cols[i] )
plt.title('A_LOADING', fontsize=15)
#plt.show()
plt.subplot(2,2,3)
#plt.plot(bp1)
plt.boxplot(bp1.mean(1))
plt.plot(b.iloc[:-1,0],ls='',marker="o",markersize=10, color="red")
plt.title('B_LOADING_1', fontsize=15)
#plt.show()
plt.subplot(2,2,4)
#plt.plot(bp2)
plt.boxplot(bp2.mean(1))
plt.plot(b.iloc[:-1,1],ls='',marker="o",markersize=10, color="red")
plt.title('B_LOADING_2', fontsize=15)
plt.savefig('40sims_Box.png')
plt.show()
In [32]:
bp1.mean(1)
Out[32]:
In [37]:
np.array(b)[:-1,1].shape
Out[37]:
In [2163]:
#dat = np.load('sims6/s1' + '/SSDF_Simu.npz')
def ab2(folder):
dat = np.load(folder + '/SSDF_Simu.npz')
#a = dat['ghg'].mean(0)
ghg =np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/ghg.csv', header=None))
a = ghg
bhg = np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/bhg.csv', header=None))
b = bhg
deltag = np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/phig.csv', header=None))
mugamg = np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/gammu.csv', header=None))
mupsig = np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/psimu.csv', header=None))
#print(b)
#a2 = np.median(ghg,0)#.median(0)
#b2 = np.median(bhg,0).reshape(-1,2)[:-1, :]
return a,b, deltag, mugamg, mupsig#, a2, b2
test = ab2('sims6/s2')
a = pd.read_csv('sims6/a.csv',index_col=0)
b = pd.read_csv('sims6/b.csv',index_col=0)
mugam = pd.read_csv('sims6/mugam.csv',index_col=0)
mupsi = pd.read_csv('sims6/mupsi.csv',index_col=0)
L = a.shape[0]
In [61]:
F = 'sims6/s1/SSDF_Simu/MCMC/'
psim = np.array(pd.read_csv(F+'psimu.csv', header=None))
gamm= np.array(pd.read_csv(F+'gammu.csv', header=None))
In [62]:
psim.mean(0)
Out[62]:
In [2166]:
#plt.plot(ap)
plt.figure(figsize=(15,15))
plt.subplot(2,2,1)
for i in range(L):
plt.plot(test[0][:,i],color=cols[i], alpha=0.5)
plt.axhline(np.array(a)[i],ls='--', color= cols[i])
plt.title('A_LOADING', fontsize=15)
#plt.show()
plt.subplot(2,2,2)
for i in range(2):
plt.plot(test[2][:,i],color=cols[i], alpha=0.5)
plt.axhline(delta[i],ls='--', color= cols[i] )
plt.title('Delta', fontsize=15)
#plt.show()
plt.subplot(2,2,3)
#plt.plot(bp1)
for i in range(L-1):
plt.plot(test[1][:,i*2],color=cols[i], alpha=0.5)
plt.axhline(np.array(np.array(b)[i,0]),ls='--', color= cols[i])
plt.title('B_LOADING_1', fontsize=15)
#plt.show()
plt.subplot(2,2,4)
#plt.plot(bp2)
for i in range(L-1):
plt.plot(test[1][:, i*2+1],color=cols[i], alpha=0.5)
plt.axhline(np.array(np.array(b)[i,1]),ls='--', color=cols[i])
plt.title('B_LOADING_2', fontsize=15)
plt.savefig('Simu_E10.png')
plt.show()
In [130]:
#plt.plot(ap)
plt.figure(figsize=(15,10))
plt.subplot(2,2,1)
for i in range(L):
plt.plot(test[0][:,i],color=cols[i], alpha=0.5)
plt.axhline(np.array(a)[i],ls='--', color= cols[i])
plt.title('A_LOADING', fontsize=15)
#plt.show()
plt.subplot(2,2,2)
for i in range(2):
plt.plot(test[2][:,i],color=cols[i], alpha=0.5)
plt.axhline(delta[i],ls='--', color= cols[i] )
plt.title('Delta', fontsize=15)
#plt.show()
plt.subplot(2,2,3)
#plt.plot(bp1)
for i in range(L-1):
plt.plot(test[1][:,i*2],color=cols[i], alpha=0.5)
plt.axhline(np.array(np.array(b)[i,0]),ls='--', color= cols[i])
plt.title('B_LOADING_1', fontsize=15)
#plt.show()
plt.subplot(2,2,4)
#plt.plot(bp2)
for i in range(L-1):
plt.plot(test[1][:, i*2+1],color=cols[i], alpha=0.5)
plt.axhline(np.array(np.array(b)[i,1]),ls='--', color=cols[i])
plt.title('B_LOADING_2', fontsize=15)
#plt.savefig('Simu_E100.png')
plt.show()
In [1417]:
test = ab2('sims6/s3')
#plt.plot(ap)
plt.figure(figsize=(15,10))
plt.subplot(2,2,1)
for i in range(L):
plt.plot(test[0][:,i],color=cols[i], alpha=0.5)
plt.axhline(np.array(a)[i],ls='--', color= cols[i])
plt.title('A_LOADING', fontsize=15)
#plt.show()
plt.subplot(2,2,2)
for i in range(2):
plt.plot(test[2][:,i],color=cols[i], alpha=0.5)
plt.axhline(delta[i],ls='--', color= cols[i] )
plt.title('Delta', fontsize=15)
#plt.show()
plt.subplot(2,2,3)
#plt.plot(bp1)
for i in range(L-1):
plt.plot(test[1][:,i*2],color=cols[i], alpha=0.5)
plt.axhline(np.array(np.array(b)[i,0]),ls='--', color= cols[i])
plt.title('B_LOADING_1', fontsize=15)
#plt.show()
plt.subplot(2,2,4)
#plt.plot(bp2)
for i in range(L-1):
plt.plot(test[1][:, i*2+1],color=cols[i], alpha=0.5)
plt.axhline(np.array(np.array(b)[i,1]),ls='--', color=cols[i])
plt.title('B_LOADING_2', fontsize=15)
"""
plt.subplot(3,2,5)
for i in range(L):
plt.plot(test[3][:, i],color=cols[i], alpha=0.5)
plt.axhline(np.array(mugam)[i],ls='--', color=cols[i])
plt.title('a_h', fontsize=15)
plt.subplot(3,2,6)
for i in range(L-1):
plt.plot(test[4][:, i],color=cols[i], alpha=0.5)
plt.axhline(np.array(mupsi)[i],ls='--', color=cols[i])
plt.title('b_h', fontsize=15)
"""
#plt.savefig('Simu_E1000.png')
plt.show()
In [478]:
#plt.plot(ap)
test = ab2('sims6/s2')
plt.figure(figsize=(15,10))
plt.subplot(2,2,1)
for i in range(L):
plt.plot(test[0][:,i],color=cols[i], alpha=0.5)
plt.axhline(np.array(a)[i],ls='--', color= cols[i])
plt.title('A_LOADING', fontsize=15)
#plt.show()
plt.subplot(2,2,2)
for i in range(2):
plt.plot(test[2][:,i],color=cols[i], alpha=0.5)
plt.axhline(delta[i],ls='--', color= cols[i] )
plt.title('Delta', fontsize=15)
#plt.show()
plt.subplot(2,2,3)
#plt.plot(bp1)
for i in range(L-1):
plt.plot(test[1][:,i*2],color=cols[i], alpha=0.5)
plt.axhline(np.array(np.array(b)[i,0]),ls='--', color= cols[i])
plt.title('B_LOADING_1', fontsize=15)
#plt.show()
plt.subplot(2,2,4)
#plt.plot(bp2)
for i in range(L-1):
plt.plot(test[1][:, i*2+1],color=cols[i], alpha=0.5)
plt.axhline(np.array(np.array(b)[i,1]),ls='--', color=cols[i])
plt.title('B_LOADING_2', fontsize=15)
"""
plt.subplot(3,2,5)
for i in range(L):
plt.plot(test[3][:, i],color=cols[i], alpha=0.5)
plt.axhline(np.array(mugam)[i],ls='--', color=cols[i])
plt.title('a_h', fontsize=15)
plt.subplot(3,2,6)
for i in range(L-1):
plt.plot(test[4][:, i],color=cols[i], alpha=0.5)
plt.axhline(np.array(mupsi)[i],ls='--', color=cols[i])
plt.title('b_h', fontsize=15)
"""
plt.savefig('Simu_E100.png')
plt.show()
In [1416]:
#plt.plot(ap)
test = ab2('sims6/s3')
plt.figure(figsize=(15,10))
plt.subplot(2,2,1)
for i in range(L):
plt.plot(test[0][:,i],color=cols[i], alpha=0.5)
plt.axhline(np.array(a)[i],ls='--', color= cols[i])
plt.title('A_LOADING', fontsize=15)
#plt.show()
plt.subplot(2,2,2)
for i in range(2):
plt.plot(test[2][:,i],color=cols[i], alpha=0.5)
plt.axhline(delta[i],ls='--', color= cols[i] )
plt.title('Delta', fontsize=15)
#plt.show()
plt.subplot(2,2,3)
#plt.plot(bp1)
for i in range(L-1):
plt.plot(test[1][:,i*2],color=cols[i], alpha=0.5)
plt.axhline(np.array(np.array(b)[i,0]),ls='--', color= cols[i])
plt.title('B_LOADING_1', fontsize=15)
#plt.show()
plt.subplot(2,2,4)
#plt.plot(bp2)
for i in range(L-1):
plt.plot(test[1][:, i*2+1],color=cols[i], alpha=0.5)
plt.axhline(np.array(np.array(b)[i,1]),ls='--', color=cols[i])
plt.title('B_LOADING_2', fontsize=15)
"""
plt.subplot(3,2,5)
for i in range(L):
plt.plot(test[3][:, i],color=cols[i], alpha=0.5)
plt.axhline(np.array(mugam)[i],ls='--', color=cols[i])
plt.title('a_h', fontsize=15)
plt.subplot(3,2,6)
for i in range(L-1):
plt.plot(test[4][:, i],color=cols[i], alpha=0.5)
plt.axhline(np.array(mupsi)[i],ls='--', color=cols[i])
plt.title('b_h', fontsize=15)
"""
plt.savefig('Simu_E10.png')
plt.show()
In [1255]:
#dat = np.load('sims6/s1' + '/SSDF_Simu.npz')
def ab3(folder):
#dat = np.load(folder + '/SSDF_Simu.npz')
#a = dat['ghg'].mean(0)
ghg =np.array(pd.read_csv(folder +'/Static_Simu/MCMC/ghg.csv', header=None))
a = ghg
bhg = np.array(pd.read_csv(folder +'/Static_Simu/MCMC/bhg.csv', header=None))
b = bhg
#deltag = np.array(pd.read_csv(folder +'/Static_Simu/MCMC/phig.csv', header=None))
L = ghg.shape[1]
print(L)
print(a.shape)
mugamg = np.array(pd.read_csv(folder +'/Static_Simu/MCMC/gammu.csv', header=None))
mupsig = np.array(pd.read_csv(folder +'/Static_Simu/MCMC/psimu.csv', header=None))
#print(b)
#a2 = np.median(ghg,0)#.median(0)
#b2 = np.median(bhg,0).reshape(-1,2)[:-1, :]
ta = pd.read_csv(folder + '/a.csv',index_col=0)
tb = pd.read_csv(folder + '/b.csv',index_col=0)
mugam = pd.read_csv(folder + '/mugam.csv',index_col=0)
mupsi = pd.read_csv(folder + '/mupsi.csv',index_col=0)
#plt.plot(ap)
plt.figure(figsize=(15,15))
plt.subplot(3,2,1)
for i in range(L):
plt.plot(a[:,i],color=cols[i], alpha=0.5)
plt.axhline(np.array(ta)[i],ls='--', color= cols[i])
plt.title('A_LOADING', fontsize=15)
#plt.show()
#plt.subplot(2,2,2)
#for i in range(2):
# plt.plot(deltag[:,i],color=cols[i], alpha=0.5)
# plt.axhline(delta[i],ls='--', color= cols[i] )
#plt.title('Delta', fontsize=15)
#plt.show()
plt.subplot(3,2,3)
#plt.plot(bp1)
for i in range(L-1):
plt.plot(b[:,i*2],color=cols[i], alpha=0.5)
plt.axhline(np.array(np.array(tb)[i,0]),ls='--', color= cols[i])
plt.title('B_LOADING_1', fontsize=15)
#plt.show()
plt.subplot(3,2,4)
#plt.plot(bp2)
for i in range(L-1):
plt.plot(b[:, i*2+1],color=cols[i], alpha=0.5)
plt.axhline(np.array(np.array(tb)[i,1]),ls='--', color=cols[i])
plt.title('B_LOADING_2', fontsize=15)
plt.subplot(3,2,5)
for i in range(L):
plt.plot(mugamg[:, i],color=cols[i], alpha=0.5)
plt.axhline(np.array(mugam)[i],ls='--', color=cols[i])
plt.title('a_h', fontsize=15)
plt.subplot(3,2,6)
for i in range(L-1):
plt.plot(mupsig[:, i],color=cols[i], alpha=0.5)
plt.axhline(np.array(mupsi)[i],ls='--', color=cols[i])
plt.title('b_h', fontsize=15)
plt.savefig('Simu_Static_Nprod10.png')
plt.show()
#plt.title('B_LOADING_1', fontsize=15)
for i in range(L-1):
plt.subplot(4,4,i+1)
plt.hist(a[:,i])
plt.axvline(np.array(np.array(ta)[i]),ls='--', color= cols[i])
#plt.savefig('Simu_S3_100_A_Loading.png')
plt.show()
for i in range(L-1):
plt.subplot(4,4,i+1)
plt.hist(b[:,i*2],color=cols[i], alpha=0.5)
plt.axvline(np.array(np.array(tb)[i,0]),ls='--', color= cols[i])
#plt.title('B_LOADING_1', fontsize=15)
#plt.savefig('Simu_S3_100_B_loading1.png')
plt.show()
for i in range(L-1):
plt.subplot(4,4,i+1)
plt.hist(b[:,i*2+1],color=cols[i], alpha=0.5)
plt.axvline(np.array(tb)[i,1],ls='--', color= cols[i])
#plt.title('A_LOADING', fontsize=15)
#plt.savefig('Simu_S3_100_B_loading2.png')
plt.show()
#return a,b, deltag, mugamg, mupsig#, a2, b2
ab3('sims7/s2')
In [1531]:
import glob
demands = glob.glob("yogurt/demand/*")
trips = []
for demand in demands:
trips.append(pd.read_csv(demand).shape[0])
In [512]:
## dat = np.load('sims6/s1' + '/SSDF_Simu.npz')
def ab4(folder, c=True):
#dat = np.load(folder + '/SSDF_Simu.npz')
#a = dat['ghg'].mean(0)
ghg =np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/ghg.csv', header=None))
a = ghg
bhg = np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/bhg.csv', header=None))
b = bhg
deltag = np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/phig.csv', header=None))
L = ghg.shape[1]
#print(L)
print(a.shape)
mugamg = np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/gammu.csv', header=None))
mupsig = np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/psimu.csv', header=None))
try:
delta = np.array(pd.read_csv(folder + '/delta_bar.csv',header=None, delimiter=' ')).reshape(-1)
except:
delta = [1, -1]
#print(delta.shape)
#a2 = np.median(ghg,0)#.median(0)
#b2 = np.median(bhg,0).reshape(-1,2)[:-1, :]
#ta = pd.read_csv(folder + '/a.csv',index_col=0)
#tb = pd.read_csv(folder + '/b.csv',index_col=0)
#mugam = pd.read_csv(folder + '/mugam.csv',index_col=0)
#mupsi = pd.read_csv(folder + '/mupsi.csv',index_col=0)
try:
ta = pd.read_csv(folder + '/a.csv',index_col=0)
tb = pd.read_csv(folder + '/b.csv',index_col=0)
mugam = pd.read_csv(folder + '/mugam.csv',index_col=0)
mupsi = pd.read_csv(folder + '/mupsi.csv',index_col=0)
except:
ta = pd.read_csv('sims6'+ '/a.csv',index_col=0)
tb = pd.read_csv('sims6' + '/b.csv',index_col=0)
mugam = pd.read_csv('sims6' + '/mugam.csv',index_col=0)
mupsi = pd.read_csv('sims6' + '/mupsi.csv',index_col=0)
#plt.plot(ap)
plt.figure(figsize=(20,15))
plt.subplot(3,2,1)
for i in range(L):
plt.plot(a[:,i],color=cols[i], alpha=0.5)
plt.axhline(np.array(ta)[i],ls='--', color= cols[i])
plt.title('A_LOADING', fontsize=15)
#plt.show()
plt.subplot(3,2,2)
for i in range(2):
plt.plot(deltag[:,i],color=cols[i], alpha=0.5)
plt.axhline(delta[i],ls='--', color= cols[i] )
plt.title('Delta', fontsize=15)
#plt.show()
plt.subplot(3,2,3)
#plt.plot(bp1)
for i in range(L-1):
plt.plot(b[:-1,i*2],color=cols[i], alpha=0.5)
plt.axhline(np.array(np.array(tb)[i,0]),ls='--', color= cols[i])
plt.title('B_LOADING_1', fontsize=15)
#plt.show()
plt.subplot(3,2,4)
#plt.plot(bp2)
for i in range(L-1):
plt.plot(b[:-1, i*2+1],color=cols[i], alpha=0.5)
plt.axhline(np.array(np.array(tb)[i,1]),ls='--', color=cols[i])
plt.title('B_LOADING_2', fontsize=15)
plt.subplot(3,2,5)
for i in range(L):
plt.plot(mugamg[:, i],color=cols[i], alpha=0.5)
plt.axhline(np.array(mugam)[i] * 1,ls='--', color=cols[i])
plt.title('a_h', fontsize=15)
plt.subplot(3,2,6)
for i in range(L-1):
plt.plot(mupsig[:-1,i],color=cols[i], alpha=0.5)
plt.axhline(np.array(mupsi)[i] * 1,ls='--', color=cols[i])
plt.title('b_h', fontsize=15)
#plt.savefig('M2_PGFIX_R1.png')
plt.show()
"""
#plt.title('B_LOADING_1', fontsize=15)
for i in range(L-1):
plt.subplot(4,4,i+1)
plt.boxplot(a[-10000:,i])
plt.scatter(1, np.array(ta)[i], color='r', marker='x', s = 60 )
#plt.axvline(np.array(np.array(ta)[i]),ls='--', color= cols[i])
#plt.savefig('Simu_S3_100_A_Loading.png')
plt.show()
for i in range(L-1):
plt.subplot(4,4,i+1)
plt.boxplot(mupsig[-10000:,i] )
plt.scatter(1, np.array(mupsi)[i] * 0, color='r', marker='x', s = 60 )
#plt.axvline(np.array(np.array(tb)[i,0]),ls='--', color= cols[i])
plt.suptitle('b_h', fontsize=15)
plt.tight_layout()
#plt.savefig('Simu_S2_E5_b_h.png')
plt.show()
for i in range(L):
plt.subplot(4,4,i+1)
plt.boxplot(mugamg[-10000:,i])
plt.scatter(1, np.array(mugam)[i] * 0, color='r', marker='x', s = 60 )
#plt.axvline(np.array(tb)[i,1],ls='--', color= cols[i])
plt.suptitle('a_h', fontsize=15)
plt.tight_layout()
#plt.savefig('Simu_S2_E5_a_h.png')
plt.show()
#return a,b, deltag, mugamg, mupsig#, a2, b2
"""
# S3, S4, S7 <- sigma 0.1
# S9, S5 <- Fix
# S2 <- var 0.1
#3
ab4('sims6/s6' , True)
#ab4('sims6/archive/archive0628/s3', True)
In [162]:
print("######### HOUSEHOLD 1000")
ab4('sims6/s2' , True)
print("######## HOUSEHOLD 400")
ab4('sims6/s3' , True)
In [455]:
bhg = np.array(pd.read_csv('sims6/s10' +'/SSDF_Simu/MCMC/bhg.csv', header=None))
b = bhg
In [51]:
b1 = b[:,[i*2 for i in range(9)]]
plt.plot(b1.sum(1))
In [54]:
np.exp(-2)
Out[54]:
In [659]:
# dat = np.load('sims6/s1' + '/SSDF_Simu.npz')
def ab4(folder, c=True):
#dat = np.load(folder + '/SSDF_Simu.npz')
#a = dat['ghg'].mean(0)
ghg =np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/ghg.csv', header=None))
a = ghg
bhg = np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/bhg.csv', header=None))
b = bhg
deltag = np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/phig.csv', header=None))
L = ghg.shape[1]
print(L)
print(a.shape)
mugamg = np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/gammu.csv', header=None))
mupsig = np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/psimu.csv', header=None))
try:
delta = np.array(pd.read_csv(folder + '/delta_bar.csv',header=None, delimiter=' ')).reshape(-1)
except:
delta = [1, -1]
#print(delta.shape)
#a2 = np.median(ghg,0)#.median(0)
#b2 = np.median(bhg,0).reshape(-1,2)[:-1, :]
#ta = pd.read_csv(folder + '/a.csv',index_col=0)
#tb = pd.read_csv(folder + '/b.csv',index_col=0)
#mugam = pd.read_csv(folder + '/mugam.csv',index_col=0)
#mupsi = pd.read_csv(folder + '/mupsi.csv',index_col=0)
try:
ta = pd.read_csv(folder + '/a.csv',index_col=0)
tb = pd.read_csv(folder + '/b.csv',index_col=0)
mugam = pd.read_csv(folder + '/mugam.csv',index_col=0)
mupsi = pd.read_csv(folder + '/mupsi.csv',index_col=0)
except:
ta = pd.read_csv('sims6'+ '/a.csv',index_col=0)
tb = pd.read_csv('sims6' + '/b.csv',index_col=0)
mugam = pd.read_csv('sims6' + '/mugam.csv',index_col=0)
mupsi = pd.read_csv('sims6' + '/mupsi.csv',index_col=0)
#plt.plot(ap)
plt.figure(figsize=(20,15))
plt.subplot(3,2,1)
plt.boxplot(a)
plt.plot(ta,ls='',marker="o",markersize=10, color="red")
#for i in range(L):
# plt.boxplot(ap[:,i],color=cols[i])
# plt.axhline(np.array(a)[i],ls='--', color= cols[i] )
plt.title('A_LOADING', fontsize=15)
#plt.show()
plt.subplot(3, 2, 3)
#plt.plot(bp1)
plt.boxplot(b[:-1,[i*2 for i in range(L-1)]])
plt.plot(tb.iloc[:-1,0],ls='',marker="o",markersize=10, color="red")
plt.title('B_LOADING_1', fontsize=15)
#plt.show()
plt.subplot(3, 2, 4)
plt.boxplot(b[:-1,[i*2 + 1 for i in range(L-1)]])
plt.plot(tb.iloc[:-1,1],ls='',marker="o",markersize=10, color="red")
plt.title('B_LOADING_2', fontsize=15)
plt.subplot(3,2,5)
plt.boxplot(mugamg)
plt.plot(mugam,ls='',marker="o",markersize=10, color="red")
plt.title('a_h', fontsize=15)
plt.subplot(3,2,6)
plt.boxplot(mupsig)
plt.plot(mupsi,ls='',marker="o",markersize=10, color="red")
plt.title('b_h', fontsize=15)
plt.savefig('S2_delta_0.3_0.1_FIX.png')
plt.show()
#plt.title('B_LOADING_1', fontsize=15)
for i in range(L-1):
plt.subplot(4,4,i+1)
plt.boxplot(a[-10000:,i])
plt.scatter(1, np.array(ta)[i], color='r', marker='x', s = 60 )
#plt.axvline(np.array(np.array(ta)[i]),ls='--', color= cols[i])
#plt.savefig('Simu_S3_100_A_Loading.png')
plt.show()
for i in range(L-1):
plt.subplot(4,4,i+1)
plt.boxplot(mupsig[-10000:,i])
plt.scatter(1, np.array(mupsi)[i], color='r', marker='x', s = 60 )
#plt.axvline(np.array(np.array(tb)[i,0]),ls='--', color= cols[i])
plt.suptitle('b_h', fontsize=15)
plt.tight_layout()
#plt.savefig('Simu_S2_E5_b_h.png')
plt.show()
for i in range(L):
plt.subplot(4,4,i+1)
plt.boxplot(mugamg[-10000:,i])
plt.scatter(1, np.array(mugam)[i], color='r', marker='x', s = 60 )
#plt.axvline(np.array(tb)[i,1],ls='--', color= cols[i])
plt.suptitle('a_h', fontsize=15)
plt.tight_layout()
#plt.savefig('Simu_S2_E5_a_h.png')
plt.show()
#return a,b, deltag, mugamg, mupsig#, a2, b2
# S3, S4, S7 <- sigma 0.1
# S9, S5 <- Fix
# S2 <- var 0.1
ab4('sims6/s2' , True)
#ab4('sims6/archive0408/s9', True)
In [2086]:
np.median(trips)
Out[2086]:
In [1534]:
trips
Out[1534]:
In [1244]:
plt.hist(trips,bins=50)
plt.text(30,60,"Means=%.2f" % np.mean(trips),fontsize=20)
plt.show()
#400
#10
#385
In [1227]:
plt.hist(trips,bins=50)
plt.text(30,60,"Means=%.2f" % np.mean(trips),fontsize=20)
plt.show()
#385
In [1220]:
Out[1220]:
In [1089]:
plt.plot(bp1)
for i in range(L):
plt.axhline(np.array(np.array(b)[i,0]),ls='--')
plt.show()
In [379]:
path = 'sims6/s2'
dat = np.load(path + '/SSDF_Simu.npz', allow_pickle=True)
In [160]:
dat['fa'][60]
#dat['betaa'][:100]
Out[160]:
In [380]:
dat['betaa']
Out[380]:
In [ ]:
In [697]:
dat['bh']
Out[697]:
In [324]:
#path = 'sims6/archive0525/s6'
path = 'sims6/s3'
dat = np.load(path + '/SSDF_Simu.npz', allow_pickle=True)
L = np.array(pd.read_csv(path +'/SSDF_Simu/MCMC/ghg.csv', header=None)).shape[0]
L = L - L %100
testpsi = dat["psi"]#(dat['psim']/L)
testgam = dat["gamma"]#(dat['gamm']/L)
truepsi = pd.read_csv(path +'/psis.csv', index_col=0)
truegam = pd.read_csv(path +'/gams.csv', index_col=0)
plt.figure(figsize=(15,10))
plt.subplot(1,2,1)
bins = 100
plt.hist(testgam.reshape(-1),bins, alpha=0.5,color='b',range=(-30,30))
plt.hist(np.array(truegam).reshape(-1),bins, alpha=0.5, color='r',range=(-30,30))
plt.legend(["Est","True"],fontsize=20)
plt.title("Gamma", fontsize=20)
#plt.show()
plt.subplot(1,2,2)
plt.hist(testpsi.reshape(-1),bins, alpha=0.5,color='b', range=(-30,30))
plt.hist(np.array(truepsi).reshape(-1),bins, alpha=0.5, color='r',range=(-30,30))
plt.title("Psi", fontsize=20)
plt.legend(["Est","True"],fontsize=20)
plt.show()
psi = likelihood * pdf(psi- g*b, 1)
psi - g* b <- 0
In [ ]:
In [642]:
dat['betam']
Out[642]:
In [1861]:
testpsi = (dat['psi'])
testgam = (dat['gamma'])
In [1192]:
np.concatenate([(testpsi.mean(0)).reshape(-1,1), np.array(truepsi.mean(0)).reshape(-1,1)],1)
Out[1192]:
In [1023]:
truepsi.head(10)
Out[1023]:
In [1862]:
truegam = pd.read_csv('sims6/s7/gams.csv', index_col=0)
In [1863]:
np.concatenate([(testgam.mean(0)).reshape(-1,1), np.array(truegam.mean(0)).reshape(-1,1)],1)
Out[1863]:
In [2314]:
#pd.DataFrame(np.concatenate( [dat['fa'], dat['ga'],dat['kt'].reshape(-1,1)],1 )).iloc[:60]
In [2308]:
estab = np.concatenate([dat['fa'], dat['ga']],1)
In [734]:
tfa = np.array(pd.read_csv( "sims6/s9/f.csv"))
tga = np.array(pd.read_csv("sims6/s9/g.csv"))
In [735]:
trueab = np.concatenate([tfa,tga],1)
In [732]:
plt.figure(figsize=(10,10))
plt.hist(diffab[:,0].reshape(-1),bins, alpha=0.5,color='b', range=(-30,30))
plt.hist(diffab[:,3].reshape(-1),bins, alpha=0.5,color='r', range=(-30,30))
plt.title("f",fontsize=20)
plt.legend(["Est","True"])
#plt.hist(np.array(truepsi).reshape(-1),bins, alpha=0.5, color='r',range=(-30,30))
plt.show()
In [2312]:
plt.figure(figsize=(10,10))
bins=200
plt.hist(diffab[:,1].reshape(-1),bins, alpha=0.5,color='b', range=(-30,30))
plt.hist(diffab[:,4].reshape(-1),bins, alpha=0.5,color='r', range=(-30,30))
plt.title("g1",fontsize=20)
plt.legend(["Est","True"])
#plt.hist(np.array(truepsi).reshape(-1),bins, alpha=0.5, color='r',range=(-30,30))
plt.show()
In [2329]:
L = np.array(pd.read_csv(path +'/SSDF_Simu/MCMC/ghg.csv', header=None)).shape[0]
L = L - L %100
beta = dat['betam'] / L
beta
In [2317]:
tbeta = np.array(pd.read_csv("sims6/s6/betas.csv"))[:,1:]
In [2320]:
plt.figure(figsize=(10,10))
plt.hist(tbeta[:,0].reshape(-1),bins, alpha=0.5,color='b', range=(-2,2))
plt.hist(beta[:,0].reshape(-1),bins, alpha=0.5,color='r', range=(-2,2))
plt.title("delta_1",fontsize=20)
plt.legend(["Est","True"])
#plt.hist(np.array(truepsi).reshape(-1),bins, alpha=0.5, color='r',range=(-30,30))
plt.show()
In [2321]:
plt.figure(figsize=(10,10))
plt.hist(tbeta[:,1].reshape(-1),bins, alpha=0.5,color='b', range=(-2,2))
plt.hist(beta[:,1].reshape(-1),bins, alpha=0.5,color='r', range=(-2,2))
plt.title("delta_2",fontsize=20)
plt.legend(["Est","True"])
#plt.hist(np.array(truepsi).reshape(-1),bins, alpha=0.5, color='r',range=(-30,30))
plt.show()
In [2322]:
plt.figure(figsize=(10,10))
plt.hist(diffab[:,2].reshape(-1),bins, alpha=0.5,color='b', range=(-30,30))
plt.hist(diffab[:,5].reshape(-1),bins, alpha=0.5,color='r', range=(-30,30))
plt.title("g2",fontsize=20)
plt.legend(["Est","True"])
#plt.hist(np.array(truepsi).reshape(-1),bins, alpha=0.5, color='r',range=(-30,30))
plt.show()
In [2323]:
diffab = np.concatenate([estab, trueab],1)
In [2324]:
print(path)
bhg = np.array(pd.read_csv(path +'/SSDF_Simu/MCMC/bhg.csv', header=None))
bm = bhg.mean(0)
tb = pd.read_csv(path + '/b.csv',index_col=0)
In [2325]:
np.concatenate([np.round(bm.reshape(-1,2),2),np.round(np.array(tb),2)],1)
#print()
Out[2325]:
In [2326]:
beta
Out[2326]:
In [2327]:
tbeta
Out[2327]:
In [448]:
#pd.DataFrame(np.round(diffab[:40],2))
diffab[:,1].reshape(1,-1).dot(diffab[:,1].reshape(-1,1))
Out[448]:
In [450]:
diffab[:,4].reshape(1,-1).dot(diffab[:,4].reshape(-1,1))
Out[450]:
In [451]:
diffab[:,0].reshape(1,-1).dot(diffab[:,0].reshape(-1,1))
Out[451]:
In [453]:
diffab[:,3].reshape(1,-1).dot(diffab[:,3].reshape(-1,1))
Out[453]:
In [764]:
### dat = np.load('sims6/s1' + '/SSDF_Simu.npz')
import re
def ab5(folder,name):
#dat = np.load(folder + '/SSDF_Simu.npz')
#a = dat['ghg'].mean(0)
ghg =np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/ghg.csv', header=None))
a = ghg
bhg = np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/bhg.csv', header=None))
b = bhg
deltag = np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/phig.csv', header=None))
L = ghg.shape[1]
print(L)
print(a.shape)
mugamg = np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/gammu.csv', header=None))
mupsig = np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/psimu.csv', header=None))
try:
delta = np.array(pd.read_csv(folder + '/delta_bar.csv',header=None, delimiter=' ')).reshape(-1)
except:
delta = [1, -1]
#print(b)
#a2 = np.median(ghg,0)#.median(0)
#b2 = np.median(bhg,0).reshape(-1,2)[:-1, :]
#ta = pd.read_csv(folder + '/a.csv',index_col=0)
#tb = pd.read_csv(folder + '/b.csv',index_col=0)
#mugam = pd.read_csv(folder + '/mugam.csv',index_col=0)
#mupsi = pd.read_csv(folder + '/mupsi.csv',index_col=0)
ta = pd.read_csv(folder + '/a.csv',index_col=0)
tb = pd.read_csv(folder + '/b.csv',index_col=0)
mugam = pd.read_csv(folder + '/mugam.csv',index_col=0)
mupsi = pd.read_csv(folder + '/mupsi.csv',index_col=0)
#plt.plot(ap)
plt.figure(figsize=(20,10))
plt.subplot(3,4,1)
for i in range(L):
plt.plot(a[:,i],color=cols[i], alpha=0.5)
plt.axhline(np.array(ta)[i],ls='--', color= cols[i])
plt.title('A_LOADING', fontsize=15)
#plt.show()
plt.subplot(3,4,2)
for i in range(2):
plt.plot(deltag[:,i],color=cols[i], alpha=0.5)
plt.axhline(delta[i],ls='--', color= cols[i] )
plt.title('Delta', fontsize=15)
#plt.show()
plt.subplot(3,4,5)
#plt.plot(bp1)
for i in range(L-1):
plt.plot(b[:,i*2],color=cols[i], alpha=0.5)
plt.axhline(np.array(np.array(tb)[i,0]),ls='--', color= cols[i])
plt.title('B_LOADING_1', fontsize=15)
#plt.show()
plt.subplot(3,4,6)
#plt.plot(bp2)
for i in range(L-1):
plt.plot(b[:, i*2+1],color=cols[i], alpha=0.5)
plt.axhline(np.array(np.array(tb)[i,1]),ls='--', color=cols[i])
plt.title('B_LOADING_2', fontsize=15)
plt.subplot(3,4,9)
for i in range(L):
plt.plot(mugamg[:, i],color=cols[i], alpha=0.5)
plt.axhline(np.array(mugam)[i],ls='--', color=cols[i])
plt.title('a_h', fontsize=15)
plt.subplot(3,4,10)
for i in range(L-1):
plt.plot(mupsig[:,i],color=cols[i], alpha=0.5)
plt.axhline(np.array(mupsi)[i],ls='--', color=cols[i])
plt.title('b_h', fontsize=15)
tmp = int( re.findall('\/s(\d)', folder)[0])
folder = 'sims6/s%i' %(tmp+1)
ghg =np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/ghg.csv', header=None))
a = ghg
bhg = np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/bhg.csv', header=None))
b = bhg
deltag = np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/phig.csv', header=None))
L = ghg.shape[1]
mugamg = np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/gammu.csv', header=None))
mupsig = np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/psimu.csv', header=None))
try:
delta = np.array(pd.read_csv(folder + '/delta_bar.csv',header=None, delimiter=' ')).reshape(-1)
except:
delta = [1, -1]
#print(b)
#a2 = np.median(ghg,0)#.median(0)
#b2 = np.median(bhg,0).reshape(-1,2)[:-1, :]
#ta = pd.read_csv(folder + '/a.csv',index_col=0)
#tb = pd.read_csv(folder + '/b.csv',index_col=0)
#mugam = pd.read_csv(folder + '/mugam.csv',index_col=0)
#mupsi = pd.read_csv(folder + '/mupsi.csv',index_col=0)
ta = pd.read_csv(folder + '/a.csv',index_col=0)
tb = pd.read_csv(folder + '/b.csv',index_col=0)
mugam = pd.read_csv(folder + '/mugam.csv',index_col=0)
mupsi = pd.read_csv(folder + '/mupsi.csv',index_col=0)
#plt.plot(ap)
plt.subplot(3,4,3)
for i in range(L):
plt.plot(a[:,i],color=cols[i], alpha=0.5)
plt.axhline(np.array(ta)[i],ls='--', color= cols[i])
plt.title('A_LOADING', fontsize=15)
#plt.show()
plt.subplot(3,4,4)
for i in range(2):
plt.plot(deltag[:,i],color=cols[i], alpha=0.5)
plt.axhline(delta[i],ls='--', color= cols[i] )
plt.title('Delta', fontsize=15)
#plt.show()
plt.subplot(3,4,7)
#plt.plot(bp1)
for i in range(L-1):
plt.plot(b[:,i*2],color=cols[i], alpha=0.5)
plt.axhline(np.array(np.array(tb)[i,0]),ls='--', color= cols[i])
plt.title('B_LOADING_1', fontsize=15)
#plt.show()
plt.subplot(3,4,8)
#plt.plot(bp2)
for i in range(L-1):
plt.plot(b[:, i*2+1],color=cols[i], alpha=0.5)
plt.axhline(np.array(np.array(tb)[i,1]),ls='--', color=cols[i])
plt.title('B_LOADING_2', fontsize=15)
plt.subplot(3,4,11)
for i in range(L):
plt.plot(mugamg[:, i],color=cols[i], alpha=0.5)
plt.axhline(np.array(mugam)[i],ls='--', color=cols[i])
plt.title('a_h', fontsize=15)
plt.subplot(3,4,12)
for i in range(L-1):
plt.plot(mupsig[:,i],color=cols[i], alpha=0.5)
plt.axhline(np.array(mupsi)[i],ls='--', color=cols[i])
plt.title('b_h', fontsize=15)
plt.suptitle(name, fontsize=20)
plt.savefig('Simu_Dynamic_S%i.png' %tmp)
plt.show()
ab5('sims6/s3','Beta Fix VS Beta No Fix')
In [788]:
ab5('sims6/s5','Beta No Fix VS Beta Fix')
In [1325]:
# Only 1 dimension for B_Loading
def ab6(folder):
#dat = np.load(folder + '/SSDF_Simu.npz')
#a = dat['ghg'].mean(0)
ghg =np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/ghg.csv', header=None))
a = ghg
bhg = np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/bhg.csv', header=None))
b = bhg
deltag = np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/phig.csv', header=None))
L = ghg.shape[1]
print(L)
print(a.shape)
mugamg = np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/gammu.csv', header=None))
mupsig = np.array(pd.read_csv(folder +'/SSDF_Simu/MCMC/psimu.csv', header=None))
delta = [0.5, -0.5]
#print(b)
#a2 = np.median(ghg,0)#.median(0)
#b2 = np.median(bhg,0).reshape(-1,2)[:-1, :]
#ta = pd.read_csv(folder + '/a.csv',index_col=0)
#tb = pd.read_csv(folder + '/b.csv',index_col=0)
#mugam = pd.read_csv(folder + '/mugam.csv',index_col=0)
#mupsi = pd.read_csv(folder + '/mupsi.csv',index_col=0)
ta = pd.read_csv(folder + '/a.csv',index_col=0)
tb = pd.read_csv(folder + '/b.csv',index_col=0)
mugam = pd.read_csv(folder + '/mugam.csv',index_col=0)
mupsi = pd.read_csv(folder + '/mupsi.csv',index_col=0)
#plt.plot(ap)
plt.figure(figsize=(15,15))
plt.subplot(3,2,1)
for i in range(L):
plt.plot(a[:,i],color=cols[i], alpha=0.5)
plt.axhline(np.array(ta)[i],ls='--', color= cols[i])
plt.title('A_LOADING', fontsize=15)
#plt.show()
plt.subplot(3,2,2)
for i in range(1):
plt.plot(deltag[:,i],color=cols[i], alpha=0.5)
plt.axhline(delta[i],ls='--', color= cols[i] )
plt.title('Delta', fontsize=15)
#plt.show()
plt.subplot(3,2,3)
#plt.plot(bp1)
for i in range(L-1):
plt.plot(b[:,i],color=cols[i], alpha=0.5)
plt.axhline(np.array(np.array(tb)[i]),ls='--', color= cols[i])
plt.title('B_LOADING_1', fontsize=15)
#plt.show()
#plt.subplot(3,2,4)
plt.subplot(3,2,5)
for i in range(L):
plt.plot(mugamg[:, i],color=cols[i], alpha=0.5)
plt.axhline(np.array(mugam)[i],ls='--', color=cols[i])
plt.title('a_h', fontsize=15)
plt.subplot(3,2,6)
for i in range(L-1):
plt.plot(mupsig[:,i],color=cols[i], alpha=0.5)
plt.axhline(np.array(mupsi)[i],ls='--', color=cols[i])
plt.title('b_h', fontsize=15)
plt.savefig('S1_B1.png')
plt.show()
#plt.title('B_LOADING_1', fontsize=15)
for i in range(L-1):
plt.subplot(4,4,i+1)
plt.boxplot(a[-10000:,i])
plt.scatter(1, np.array(ta)[i], color='r', marker='x', s = 60 )
#plt.axvline(np.array(np.array(ta)[i]),ls='--', color= cols[i])
#plt.savefig('Simu_S3_100_A_Loading.png')
plt.show()
for i in range(L-1):
plt.subplot(4,4,i+1)
plt.boxplot(mupsig[-10000:,i])
plt.scatter(1, np.array(mupsi)[i], color='r', marker='x', s = 60 )
#plt.axvline(np.array(np.array(tb)[i,0]),ls='--', color= cols[i])
plt.suptitle('b_h', fontsize=15)
plt.tight_layout()
#plt.savefig('Simu_S2_E5_b_h.png')
plt.show()
for i in range(L-1):
plt.subplot(4,4,i+1)
plt.boxplot(mugamg[-10000:,i])
plt.scatter(1, np.array(mugam)[i], color='r', marker='x', s = 60 )
#plt.axvline(np.array(tb)[i,1],ls='--', color= cols[i])
plt.suptitle('a_h', fontsize=15)
plt.tight_layout()
#plt.savefig('Simu_S2_E5_a_h.png')
plt.show()
#return a,b, deltag, mugamg, mupsig#, a2, b2
ab6('sims8/s1')
In [800]:
from scipy.linalg import cholesky
test = np.array([[1, 0.5], [0.5, 2]])
display(test)
cholesky(test,True)
Out[800]:
In [661]:
from Utils.Utils import *
In [788]:
test2 =test[np.tril_indices(test.shape[0], 0)]
In [789]:
test2
Out[789]:
In [790]:
test3 = xpnd(test2)
test3
Out[790]:
In [791]:
chol(test3)
Out[791]:
In [829]:
t1 = []
t2 = []
for i in range(10000):
t1.append(mvn(test))
t2.append(cholesky(test,True).dot(np.random.standard_normal(2) ))
In [825]:
np.random.standard_normal(5)
Out[825]:
In [818]:
t2
Out[818]:
In [1758]:
w = np.eye(3)
g = np.eye(3)
w[1:,1:] = np.eye(2) * 0
test = g.dot(w).dot(g.T)
test
Out[1758]:
In [772]:
np.linalg.inv(np.eye(2))
Out[772]:
In [253]:
dat.head()
Out[253]:
In [255]:
dat = pd.read_csv("/home/practice_data/folder2/test_file.gz", compression='gzip', sep='\t')
dat
In [619]:
import gzip
FILES = glob.glob('/home/dacdata/2021/dataset/daclog/*')
stop = 100
count = 0
dat = []
def makedat(file, test=False, count = 100):
with gzip.open(file,'r') as file:
count = 0
for line in file:
l = line.decode('utf-8').split(',')
ll = len(l)
dat.append(l[:13]) #前の13列のみ取る
if count == stop and test:
break
count += 1
return pd.DataFrame(dat)
makedat(FILES[0], True)
Out[619]:
In [8]:
import gzip
import pandas as pd
import glob
import gzip
FILES = glob.glob('/home/dacdata/2021/dataset/daclog/*')
def makedat(file, test=False, stop = 100):
dat = []
with gzip.open(file,'r') as file:
count = 0
for line in file:
l = line.decode('utf-8').split(',')
ll = len(l)
dat.append(l[:13]) #前の13列のみ取る
if count == stop and test:
break
count += 1
return pd.DataFrame(dat)
makedat(FILES[0], True)
Out[8]:
In [ ]:
#cols = ["view_datetime", "user_ID", "purl_subdomain", "purl_base", "purl_path1", "purl_path2", "browser", "browser_version", "os", "os_version", "pc_sd", "ip_address"]
import re
data = pd.read_csv(FILES[0], sep='[^(KHTML)]\,', header=None)
In [622]:
data.head()
Out[622]:
In [ ]:
In [516]:
In [508]:
import pandas as pd
In [276]:
import argparse
import sys
import csv
import glob
import gzip
import tracemalloc
def makedat(file, path, test=False, stop=100):
print("Start: makedat", file=sys.stderr)
with open(path,'w',newline='') as f: #Initialize
pass
with gzip.open(file,'r') as file:
with open(path,'a',newline='') as f: #Append Mode
count = 0
#dat = []
writer = csv.writer(f)
for line in file:
l = line.decode('utf-8').split(',')
l[12] = ','.join(l[12:]) #前の13列以降を結合する
#dat.append(l[:13])
writer.writerow(l)
if count == stop and test:
break
count += 1
print("End: makedat", file=sys.stderr)
#return pd.DataFrame(dat, columns=COLS_DACLOG)
daclog_files = glob.glob('/home/dacdata/2021/dataset/daclog/*')
makedat(daclog_files[0], '/home/li/test.csv',True)
In [517]:
test = pd.read_csv(daclog_files[0], usecols=[*range(13)], header=None)
In [524]:
test = pd.read_csv('/home/dacdata/2021/dataset/prism3/mst_prism3_url_map_000.gz')
In [526]:
test.head(100)
Out[526]:
In [528]:
test2 = pd.read_csv('/home/dacdata/2021/dataset/prism3/mst_prism3_v2_purl_base_map_000.gz')
test2.head()
Out[528]:
In [523]:
Out[523]:
In [515]:
#pd.read_csv('/home/dacdata/2021/appendix/PRISM3_mapping_master.gz')
Out[515]:
In [ ]:
In [282]:
def count_missing_data(file_path, debug=False):
print("Start: count_missing_data", file=sys.stderr)
if debug:
tracemalloc.start()
#df = makedat(file_path)
total_missing = 0
with gzip.open(file_path,'r') as file:
count = 0
dat = []
for line in file:
l = line.decode('utf-8').split(',')
l[12] = ','.join(l[12:]) #前の13列以降を結合する
total_missing += (pd.DataFrame(l).isnull() | pd.DataFrame(l)== "").sum()
return total_missing
In [283]:
count_missing_data(daclog_files[0])